# encoding: utf-8
# author: 侯佳涛
# file_name: 3.tieba_test
# create_time: 2021/5/23, 下午12:48
import requests
import json
from lxml import etree


class TieBa:
    def __init__(self, name):
        self.url = 'https://tieba.baidu.com/f?kw={}'.format(name)
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
        }
        self.proxies = {
            'http': 'http://106.45.104.212:3256',
            # 'https': 'https://123.171.42.68:3256'
        }

    # @staticmethod
    def get_data(self, url, headers):
        # 发送请求获取相应
        response = requests.get(url, headers=headers, proxies=self.proxies)
        # 获取html的Element对象
        response_data = response.content.decode().replace('<!--', '').replace('-->', '')
        # with open('./demo.html', 'w') as f:
        #     f.write(response_data)
        # raise Exception('ting')
        html = etree.HTML(response_data)
        # 获取a标签的Element对象集合
        a_el_list = html.xpath('//*[@id="thread_list"]/li[position()>=2]/div/div[2]/div[1]/div[1]/a')
        # print(a_el_list)
        # 获取title和href
        return_data = []
        for el in a_el_list:
            try:
                title = el.xpath('./@title')[0]
                href = 'https://tieba.baidu.com' + el.xpath('./@href')[0]
                # print(title, href)
                return_data.append({title: href})
            except:
                # print('执行这里了？')
                continue

        # print(return_data)
        # print(len(return_data))
        # raise Exception('ting')
        # 获取下一页链接
        # //*[@id="frs_list_pager"]/a[10] 第一页到第二页
        # //*[@id="frs_list_pager"]/a[12] 第二页到第三页
        try:
            next_url = 'https:' + html.xpath('//*[@id="frs_list_pager"]/a[contains(text(), "下一页")]/@href')[0]
        except:
            next_url = None

        # 返回数据
        return return_data, next_url

    @staticmethod
    def parse_data(data):
        json.dump(data, open('save_tieba_dengjiaxian.json', 'w', encoding='utf-8'), ensure_ascii=False)

    def run(self):
        # 获取url
        next_url = self.url
        # 创建请求头
        headers = self.headers

        # 发送请求，获取输出
        save_data = []
        while True:
            data, next_url = self.get_data(next_url, headers)
            save_data.extend(data)

            if not next_url:
                break

        # 处理数据
        self.parse_data(save_data)

        print(save_data)
        print(len(save_data))


if __name__ == '__main__':
    tieba = TieBa('袁隆平')
    tieba.run()

    # tieba = TieBa('邓稼先')
    # tieba.run()
    data = json.load(open('./save_tieba_dengjiaxian.json', encoding='utf-8'))
    print(len(data))
    print(data)

